<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>尚硅谷大数据之Spark基础解析(一) | Cquang博客</title><meta name="keywords" content="大数据,spark,spark运行模式"><meta name="author" content="Cai XianQuan"><meta name="copyright" content="Cai XianQuan"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta http-equiv="Cache-Control" content="no-transform"><meta http-equiv="Cache-Control" content="no-siteapp"><meta name="description" content="从本章开始是学习尚硅谷的大数据技术之Spark---该笔记主要是了解Spark概述、运行模式以及安装方式。">
<meta property="og:type" content="article">
<meta property="og:title" content="尚硅谷大数据之Spark基础解析(一)">
<meta property="og:url" content="https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/index.html">
<meta property="og:site_name" content="Cquang博客">
<meta property="og:description" content="从本章开始是学习尚硅谷的大数据技术之Spark---该笔记主要是了解Spark概述、运行模式以及安装方式。">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90%E9%A6%96%E9%A1%B5.jpg">
<meta property="article:published_time" content="2020-08-08T09:22:38.000Z">
<meta property="article:modified_time" content="2020-12-05T16:34:59.000Z">
<meta property="article:author" content="Cai XianQuan">
<meta property="article:tag" content="大数据">
<meta property="article:tag" content="spark">
<meta property="article:tag" content="spark运行模式">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90%E9%A6%96%E9%A1%B5.jpg"><link rel="shortcut icon" href="/blog/img/favicon1.png"><link rel="canonical" href="https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//fonts.googleapis.com" crossorigin="crossorigin"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/blog/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css"><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Titillium+Web"><script>var GLOBAL_CONFIG = { 
  root: '/blog/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  ClickShowText: undefined,
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#121212","position":"bottom-left"},
  justifiedGallery: {
    js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
    css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: false
};

var saveToLocal = {
  set: function setWithExpiry(key, value, ttl) {
    const now = new Date()
    const expiryDay = ttl * 86400000
    const item = {
      value: value,
      expiry: now.getTime() + expiryDay,
    }
    localStorage.setItem(key, JSON.stringify(item))
  },

  get: function getWithExpiry(key) {
    const itemStr = localStorage.getItem(key)

    if (!itemStr) {
      return undefined
    }
    const item = JSON.parse(itemStr)
    const now = new Date()

    if (now.getTime() > item.expiry) {
      localStorage.removeItem(key)
      return undefined
    }
    return item.value
  }
}</script><script id="config_change">var GLOBAL_CONFIG_SITE = { 
  isPost: true,
  isHome: false,
  isHighlightShrink: true,
  isToc: true,
  postUpdate: '2020-12-06 00:34:59'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(function () {  window.activateDarkMode = function () {
    document.documentElement.setAttribute('data-theme', 'dark')
    if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
    }
  }
  window.activateLightMode = function () {
    document.documentElement.setAttribute('data-theme', 'light')
   if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
    }
  }
  const autoChangeMode = 'false'
  const t = saveToLocal.get('theme')
  if (autoChangeMode === '1') {
    const isDarkMode = window.matchMedia('(prefers-color-scheme: dark)').matches
    const isLightMode = window.matchMedia('(prefers-color-scheme: light)').matches
    const isNotSpecified = window.matchMedia('(prefers-color-scheme: no-preference)').matches
    const hasNoSupport = !isDarkMode && !isLightMode && !isNotSpecified
    if (t === undefined) {
      if (isLightMode) activateLightMode()
      else if (isDarkMode) activateDarkMode()
      else if (isNotSpecified || hasNoSupport) {
        const now = new Date()
        const hour = now.getHours()
        const isNight = hour <= 6 || hour >= 18
        isNight ? activateDarkMode() : activateLightMode()
      }
      window.matchMedia('(prefers-color-scheme: dark)').addListener(function (e) {
        if (saveToLocal.get('theme') === undefined) {
          e.matches ? activateDarkMode() : activateLightMode()
        }
      })
    } else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else if (autoChangeMode === '2') {
    const now = new Date()
    const hour = now.getHours()
    const isNight = hour <= 6 || hour >= 18
    if (t === undefined) isNight ? activateDarkMode() : activateLightMode()
    else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else {
    if (t === 'dark') activateDarkMode()
    else if (t === 'light') activateLightMode()
  }const asideStatus = saveToLocal.get('aside-status')
if (asideStatus !== undefined) {
   if (asideStatus === 'hide') {
     document.documentElement.classList.add('hide-aside')
   } else {
     document.documentElement.classList.remove('hide-aside')
   }
}})()</script><link rel="stylesheet" href="/css/VolantisTags.css"><meta name="generator" content="Hexo 5.3.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="author-avatar"><img class="avatar-img" src="/blog/img/%E5%A4%B4%E5%83%8F.gif" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">22</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">33</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 享受时光</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/blog/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li></ul></div></div></div></div><div id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url(/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90%E9%A6%96%E9%A1%B5.jpg)"><nav id="nav"><span id="blog_name"><a id="site-name" href="/blog/">Cquang博客</a></span><span id="menus"><div id="search_button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 享受时光</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/blog/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li></ul></div></div><span class="close" id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></span></span></nav><div id="post-info"><h1 class="post-title">尚硅谷大数据之Spark基础解析(一)</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2020-08-08T09:22:38.000Z" title="发表于 2020-08-08 17:22:38">2020-08-08</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2020-12-05T16:34:59.000Z" title="更新于 2020-12-06 00:34:59">2020-12-06</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%AD%A6%E4%B9%A0/">大数据学习</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%AD%A6%E4%B9%A0/Spark/">Spark</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%AD%A6%E4%B9%A0/Spark/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%AD%A6%E4%B9%A0/">尚硅谷大数据之Spark学习</a></span></div><div class="meta-secondline"> <span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">4.8k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>19分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="Spark概述"><a href="#Spark概述" class="headerlink" title="Spark概述"></a>Spark概述</h1><h2 id="什么是Spark"><a href="#什么是Spark" class="headerlink" title="什么是Spark"></a>什么是Spark</h2><h3 id="定义"><a href="#定义" class="headerlink" title="定义"></a>定义</h3><ul>
<li>Spark是一种基于<font color=red size=3><strong>内存</strong></font>的快速、通用、可扩展的大数据分析引擎。</li>
</ul>
<h3 id="历史"><a href="#历史" class="headerlink" title="历史"></a>历史</h3><p><img src="1%E3%80%81Hadoop%E5%8E%86%E5%8F%B2.png" alt="Hadoop历史"></p>
<ul>
<li>MR的缺点．</li>
</ul>
<ol>
<li>mr基于数据集的计算，所以面向数据</li>
<li>基本运算规则从存储介质中获取（采集）数据，然后进行计算，最后将结果存储到介质中，所以主要应用于一次性计算，不适合于数据挖掘和机器学习这样的迭代计算和图形挖掘计算。</li>
<li>MR基于文件存储介质的操作，所以性能非常的慢</li>
<li>MR和hadoop紧密耦合在一起，无法动态替换</li>
</ol>
<ul>
<li>2009年诞生于加州大学伯克利分校AMPLab，项目采用<font color=red size=3><strong>Scala编写</strong></font>。</li>
<li>2010年开源;</li>
<li>2013年6月成为Apache孵化项目(基于Hadoop问题，将资源和任务调度分开)</li>
<li>2014年2月成为Apache顶级项目。<br><img src="2%E3%80%81Spark%E5%8E%86%E5%8F%B2.png" alt="Spark历史"></li>
<li>Yarn解决了MR和hadoop的耦合性过高问题，将MR作为yarn上的一个热插拔的组件。可以与其他组件一起使用。</li>
<li>Spark将资源与计算分开来，降低耦合性，而没考虑是否能够替换组件问题。Yarn可以支持Spark的计算。</li>
<li>当使用Yarn和Spark时，master和worker就使用Yarn上的RM和NM</li>
<li>Yarn是为了其他框架也可以使用，所以用了container，兼容性更好</li>
<li>spark的资源调度框架就是为了自己使用，所以没有必要用container，效率更高</li>
<li>一般都是使用HDFS作为存储，Yarn作为资源的管理和调度，spark作为计算</li>
</ul>
<h3 id="Hadoop-MR框架和Spark框架"><a href="#Hadoop-MR框架和Spark框架" class="headerlink" title="Hadoop MR框架和Spark框架"></a>Hadoop MR框架和Spark框架</h3><ul>
<li>spark框架中间结果不落盘，但是shuffle操作还是会落盘</li>
<li>支持迭代式计算，图形计算<br>![spark框架和Hadoop MR框架](36、spark框架和Hadoop MR框架.png)</li>
</ul>
<h2 id="Spark内置模块"><a href="#Spark内置模块" class="headerlink" title="Spark内置模块"></a>Spark内置模块</h2><p><img src="3%E3%80%81spark%E5%86%85%E7%BD%AE%E6%A8%A1%E5%9D%97.png" alt="spark内置模块"></p>
<ul>
<li><p><font color=red size=3><strong>Spark Core</strong></font>：实现了Spark的基本功能，包含任务调度、内存管理、错误恢复、与存储系统交互等模块。Spark Core中还包含了对弹性分布式数据集(Resilient Distributed DataSet，简称RDD)的API定义。 </p>
</li>
<li><p><font color=red size=3><strong>Spark SQL</strong></font>：是Spark用来操作结构化数据的程序包。通过Spark SQL，我们可以使用 SQL或者Apache Hive版本的SQL方言(HQL)来查询数据。Spark SQL支持多种数据源，比如Hive表、Parquet以及JSON等。 </p>
</li>
<li><p><font color=red size=3><strong>Spark Streaming</strong></font>：是Spark提供的对实时数据进行流式计算的组件。提供了用来操作数据流的API，并且与Spark Core中的 RDD API高度对应。 </p>
</li>
<li><p><font color=red size=3><strong>Spark MLlib</strong></font>：提供常见的机器学习(ML)功能的程序库。包括分类、回归、聚类、协同过滤等，还提供了模型评估、数据 导入等额外的支持功能。 </p>
</li>
<li><p><font color=red size=3><strong>Spark GraphX</strong></font>：主要用于图形并行计算和图挖掘系统的组件。</p>
</li>
<li><p><font color=red size=3><strong>集群管理器</strong></font>：Spark 设计为可以高效地在一个计算节点到数千个计算节点之间伸缩计 算。为了实现这样的要求，同时获得最大灵活性，Spark支持在各种集群管理器(Cluster Manager)上运行，包括Hadoop YARN、Apache Mesos【国外使用比较多】，以及Spark自带的一个简易调度器，叫作独立调度器。 </p>
<p>Spark得到了众多大数据公司的支持，这些公司包括Hortonworks、IBM、Intel、Cloudera、MapR、Pivotal、百度、阿里、腾讯、京东、携程、优酷土豆。当前百度的Spark已应用于大搜索、直达号、百度大数据等业务；阿里利用GraphX构建了大规模的图计算和图挖掘系统，实现了很多生产系统的推荐算法；腾讯Spark集群达到8000台的规模，是当前已知的世界上最大的Spark集群。</p>
</li>
</ul>
<h2 id="Spark特点"><a href="#Spark特点" class="headerlink" title="Spark特点"></a>Spark特点</h2><ol>
<li><font color=red size=3><strong>快</strong></font>：与Hadoop的MapReduce相比，Spark基于<font color=red size=3><strong>内存</strong></font>的运算要<font color=red size=3><strong>快100倍以上</strong></font>，基于<font color=red size=3><strong>硬盘</strong></font>的运算也要<font color=red size=3><strong>快10倍</strong></font>以上。Spark实现了高效的DAG（有向无环图）执行引擎，可以通过基于内存来高效处理数据流。计算的中间结果是存在于内存中的。</li>
<li><font color=red size=3><strong>易用</strong></font>：Spark支持<font color=red size=3><strong>Java、Python和Scala的API</strong></font>，还支持超过80种高级算法，使用户可以快速构建不同的应用。而且Spark支持交互式的Python和Scala的Shell，可以非常方便地在这些Shell中使用Spark集群来验证解决问题的方法。</li>
<li><font color=red size=3><strong>通用</strong></font>：Spark提供了统一的解决方案。Spark可以用于，交互式查询（<font color=red size=3><strong>Spark SQL</strong></font>）、实时流处理（<font color=red size=3><strong>Spark Streaming</strong></font>）、机器学习（<font color=red size=3><strong>Spark MLlib</strong></font>）和图计算（<font color=red size=3><strong>GraphX</strong></font>）。这些不同类型的处理都可以在同一个应用中无缝使用。减少了开发和维护的人力成本和部署平台的物力成本。</li>
<li><font color=red size=3><strong>兼容性</strong></font>：Spark可以非常方便地与其他的开源产品进行融合。比如，Spark<font color=red size=3><strong>可以使用Hadoop的YARN</strong></font>和Apache Mesos作为它的资源管理和调度器，并且可以<font color=red size=3><strong>处理所有Hadoop支持的数据</strong></font>，包括HDFS、HBase等。这对于已经部署Hadoop集群的用户特别重要，因为不需要做任何数据迁移就可以使用Spark的强大处理能力。</li>
</ol>
<h1 id="Spark运行模式"><a href="#Spark运行模式" class="headerlink" title="Spark运行模式"></a>Spark运行模式</h1><h2 id="Spark安装地址"><a href="#Spark安装地址" class="headerlink" title="Spark安装地址"></a>Spark安装地址</h2><ul>
<li>官网地址<br><a target="_blank" rel="noopener" href="http://spark.apache.org/">http://spark.apache.org/</a></li>
<li>文档查看地址<br><a target="_blank" rel="noopener" href="https://spark.apache.org/docs/2.1.1/">https://spark.apache.org/docs/2.1.1/</a></li>
<li>下载地址<br><a target="_blank" rel="noopener" href="https://spark.apache.org/downloads.html">https://spark.apache.org/downloads.html</a></li>
</ul>
<h2 id="重要角色"><a href="#重要角色" class="headerlink" title="重要角色"></a>重要角色</h2><h3 id="Driver（驱动器）【管理者】"><a href="#Driver（驱动器）【管理者】" class="headerlink" title="Driver（驱动器）【管理者】"></a>Driver（驱动器）【管理者】</h3><p>Spark的驱动器是执行开发程序中的main方法的进程。它负责开发人员编写的用来创建SparkContext、创建RDD，以及进行RDD的转化操作和行动操作代码的执行。如果你是用spark shell，那么当你启动Spark shell的时候，系统后台自启了一个Spark驱动器程序，就是在Spark shell中预加载的一个叫作 sc的SparkContext对象。如果驱动器程序终止，那么Spark应用也就结束了。主要负责：</p>
<ol>
<li>把用户程序转为作业（JOB）</li>
<li>跟踪Executor的运行状况</li>
<li>为执行器节点调度任务</li>
<li>UI展示应用运行状况</li>
</ol>
<h3 id="Executor（执行器）"><a href="#Executor（执行器）" class="headerlink" title="Executor（执行器）"></a>Executor（执行器）</h3><p>Spark Executor是一个工作进程，负责在 Spark 作业中运行任务，任务间相互独立。Spark 应用启动时，Executor节点被同时启动，并且始终伴随着整个 Spark 应用的生命周期而存在。如果有Executor节点发生了故障或崩溃，Spark 应用也可以继续执行，会将出错节点上的任务调度到其他Executor节点上继续运行。主要负责：</p>
<ol>
<li>负责运行组成 Spark 应用的任务，并将结果返回给驱动器进程；</li>
<li>通过自身的块管理器（Block Manager）为用户程序中要求缓存的RDD提供内存式存储。RDD是直接缓存在Executor进程内的，因此任务可以在运行时充分利用缓存数据加速运算。</li>
</ol>
<h2 id="Local模式"><a href="#Local模式" class="headerlink" title="Local模式"></a>Local模式</h2><h3 id="概述"><a href="#概述" class="headerlink" title="概述"></a>概述</h3><ul>
<li><p>Local模式就是运行在<font color=red size=3><strong>一台计算机上的模式</strong></font>，通常就是用于在本机上练手和测试。它可以通过以下集中方式设置Master。</p>
</li>
<li><p>local: 所有计算都运行在一个线程当中，没有任何并行计算，通常我们在本机执行一些测试代码，或者练手，就用这种模式;</p>
</li>
<li><p>local[K]: 指定使用几个线程来运行计算，比如local[4]就是运行4个Worker线程。<font color=red size=3><strong>通常我们的Cpu有几个Core，就指定几个线程</strong></font>，最大化利用Cpu的计算能力;</p>
</li>
<li><p>local[*]: 这种模式直接帮你按照Cpu最多Cores来设置线程数了。</p>
</li>
</ul>
<h3 id="安装使用"><a href="#安装使用" class="headerlink" title="安装使用"></a>安装使用</h3><ul>
<li><p>上传并解压spark安装包</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 sorfware]$ tar -zxvf spark-2.1.1-bin-hadoop2.7.tgz -C /opt/module/</span><br><span class="line">[caixianquan@hadoop102 module]$ mv spark-2.1.1-bin-hadoop2.7 spark</span><br></pre></td></tr></table></figure></li>
<li><p>官方求PI案例<br>【反斜杠 \ 表示连接，太长换行】</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ bin/spark-submit \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--master local[2] \</span><br><span class="line">./examples/jars/spark-examples_2.11-2.1.1.jar \</span><br><span class="line">100</span><br></pre></td></tr></table></figure>
<div class="note info flat"><p>（1）基本语法</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">bin/spark-submit \</span><br><span class="line">--class &lt;main-class&gt;</span><br><span class="line">--master &lt;master-url&gt; \</span><br><span class="line">--deploy-mode &lt;deploy-mode&gt; \</span><br><span class="line">--conf &lt;key&gt;=&lt;value&gt; \</span><br><span class="line"><span class="meta">#</span><span class="bash"> other options</span></span><br><span class="line">&lt;application-jar&gt; \</span><br><span class="line">[application-arguments]</span><br></pre></td></tr></table></figure>
<p>（2）参数说明：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">--master 指定Master的地址，默认为Local</span><br><span class="line">--class: 表示要执行的的主类 (如 org.apache.spark.examples.SparkPi)</span><br><span class="line">--deploy-mode: 是否发布你的驱动到worker节点(cluster) 或者作为一个本地客户端 (client) (default: client)*</span><br><span class="line">--conf: 任意的Spark配置属性， 格式key=value. 如果值包含空格，可以加引号“key=value” </span><br><span class="line">application-jar: 打包好的应用jar,包含依赖. 这个URL在集群中全局可见。 比如hdfs:// 共享存储系统， 如果是 file:// path， 那么所有的节点的path都包含同样的jar</span><br><span class="line">application-arguments: 传给main()方法的参数</span><br><span class="line">--executor-memory 1G 指定每个executor可用内存为1G</span><br><span class="line">--total-executor-cores 2 指定每个executor使用的cup核数为2个</span><br></pre></td></tr></table></figure></div>
</li>
<li><p>结果展示<br>该算法是利用蒙特·卡罗算法求PI【<font color=red size=3><strong>后面的100是传入的参数，计算100次</strong></font>】<br><img src="4%E3%80%81%E8%92%99%E7%89%B9%C2%B7%E5%8D%A1%E7%BD%97%E7%AE%97%E6%B3%95.png" alt="蒙特·卡罗算法"></p>
</li>
<li><p>准备文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ mkdir input</span><br></pre></td></tr></table></figure>
<p>在input下创建文件1.txt和2.txt，并输入以下内容</p>
<table>
<thead>
<tr>
<th align="left">1.txt</th>
<th align="left">2.txt</th>
</tr>
</thead>
<tbody><tr>
<td align="left">Hello World<br>Hello Spark</td>
<td align="left">Hello Spark</td>
</tr>
</tbody></table>
</li>
<li><p>启动spark-shell</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ bin/spark-shell</span><br><span class="line">Using Spark&#x27;s default log4j profile: org/apache/spark/log4j-defaults.properties</span><br><span class="line">Setting default log level to &quot;WARN&quot;.</span><br><span class="line">To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).</span><br><span class="line">18/09/29 08:50:52 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">18/09/29 08:50:58 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException</span><br><span class="line">Spark context Web UI available at http://192.168.9.102:4040</span><br><span class="line">Spark context available as &#x27;sc&#x27; (master = local[*], app id = local-1538182253312).</span><br><span class="line">Spark session available as &#x27;spark&#x27;.</span><br><span class="line">Welcome to</span><br><span class="line">      ____              __</span><br><span class="line">     / __/__  ___ _____/ /__</span><br><span class="line">    _\ \/ _ \/ _ `/ __/  &#x27;_/</span><br><span class="line">   /___/ .__/\_,_/_/ /_/\_\   version 2.1.1</span><br><span class="line">      /_/</span><br><span class="line">         </span><br><span class="line">Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_144)</span><br><span class="line">Type in expressions to have them evaluated.</span><br><span class="line">Type :help for more information.</span><br><span class="line"></span><br><span class="line"><span class="meta">scala&gt;</span></span><br></pre></td></tr></table></figure>
<div class="note primary flat"><p>注意：sc是SparkCore程序的入口；spark是SparkSQL程序入口；master = local[*]表示本地模式运行。</p>
</div>
<p>开启另一个CRD窗口</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ jps</span><br><span class="line">3627 SparkSubmit</span><br><span class="line">4047 Jps</span><br></pre></td></tr></table></figure>
<p>可登录hadoop102:4040查看程序运行<br><img src="5%E3%80%81%E6%9F%A5%E7%9C%8B%E7%A8%8B%E5%BA%8F%E8%BF%90%E8%A1%8C.png" alt="查看程序运行"></p>
</li>
</ul>
<h3 id="运行WordCount程序"><a href="#运行WordCount程序" class="headerlink" title="运行WordCount程序"></a>运行WordCount程序</h3><figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">scala&gt;sc.textFile(<span class="string">&quot;input&quot;</span>).flatMap(_.split(<span class="string">&quot; &quot;</span>)).map((_,<span class="number">1</span>)).reduceByKey(_+_).collect</span><br><span class="line">res0: <span class="type">Array</span>[(<span class="type">String</span>, <span class="type">Int</span>)] = <span class="type">Array</span>((<span class="type">Hello</span>,<span class="number">3</span>), (<span class="type">World</span>,<span class="number">1</span>), (<span class="type">Scala</span>,<span class="number">1</span>), (<span class="type">Spark</span>,<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">scala&gt;</span><br></pre></td></tr></table></figure>
<p><font color=red size=3><strong>可登录hadoop102:4040查看程序运行</strong></font><br><img src="6%E3%80%81%E6%9F%A5%E7%9C%8BWordCount%E7%A8%8B%E5%BA%8F.png" alt="查看WordCount程序"></p>
<ul>
<li>WordCount程序分析</li>
</ul>
<p>提交任务分析：<br><img src="7%E3%80%81WordCount%E7%A8%8B%E5%BA%8F%E5%88%86%E6%9E%90.png" alt="WordCount程序分析"><br>数据流分析：</p>
<p>textFile(“input”)：读取本地文件input文件夹数据；<br>flatMap(<em>.split(“ “))：压平操作，按照空格分割符将一行数据映射成一个个单词；<br>map((</em>,1))：对每一个元素操作，将单词映射为元组；<br>reduceByKey(<em>+</em>)：按照key将值进行聚合，相加；<br>collect：将数据收集到Driver端展示。<br><img src="8%E3%80%81wordCount%E4%BB%A3%E7%A0%81%E5%88%86%E6%9E%90.png" alt="wordCount代码分析"></p>
<h2 id="Standalone模式"><a href="#Standalone模式" class="headerlink" title="Standalone模式"></a>Standalone模式</h2><h3 id="概述-1"><a href="#概述-1" class="headerlink" title="概述"></a>概述</h3><p>构建一个由Master+Slave构成的Spark集群，Spark运行在集群中。<br><img src="9%E3%80%81Standalone%E8%BF%90%E8%A1%8C%E6%A8%A1%E5%BC%8F.png" alt="Standalone运行模式"></p>
<h3 id="安装使用-1"><a href="#安装使用-1" class="headerlink" title="安装使用"></a>安装使用</h3><ul>
<li><p>进入spark安装目录下的conf文件夹</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 module]$ cd spark/conf/</span><br></pre></td></tr></table></figure></li>
<li><p>修改配置文件名称</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ mv slaves.template slaves</span><br><span class="line">[atguigu@hadoop102 conf]$ mv spark-env.sh.template spark-env.sh</span><br></pre></td></tr></table></figure></li>
<li><p>修改slave文件，添加work节点：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ vim slaves</span><br><span class="line"></span><br><span class="line">hadoop102</span><br><span class="line">hadoop103</span><br><span class="line">hadoop104</span><br></pre></td></tr></table></figure></li>
<li><p>修改spark-env.sh文件，添加如下配置：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ vim spark-env.sh</span><br><span class="line"></span><br><span class="line">SPARK_MASTER_HOST=hadoop102</span><br><span class="line">SPARK_MASTER_PORT=7077</span><br></pre></td></tr></table></figure></li>
<li><p>分发spark包</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 module]$ xsync spark/</span><br></pre></td></tr></table></figure></li>
<li><p>启动</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 spark]$ sbin/start-all.sh</span><br><span class="line">[atguigu@hadoop102 spark]$ util.sh </span><br><span class="line">================atguigu@hadoop102================</span><br><span class="line">3330 Jps</span><br><span class="line">3238 Worker</span><br><span class="line">3163 Master</span><br><span class="line">================atguigu@hadoop103================</span><br><span class="line">2966 Jps</span><br><span class="line">2908 Worker</span><br><span class="line">================atguigu@hadoop104================</span><br><span class="line">2978 Worker</span><br><span class="line">3036 Jps</span><br></pre></td></tr></table></figure>
<p><font color=red size=3><strong>网页查看：hadoop102:8080</strong></font></p>
<div class="note primary flat"><p>注意：如果遇到 “JAVA_HOME not set” 异常，可以在sbin目录下的spark-config.sh 文件中加入如下配置：<br>export JAVA_HOME=XXXX</p>
</div>
</li>
<li><p>官方求PI案例</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 spark]$ bin/spark-submit \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--master spark://hadoop102:7077 \</span><br><span class="line">--executor-memory 1G \</span><br><span class="line">--total-executor-cores 2 \</span><br><span class="line">./examples/jars/spark-examples_2.11-2.1.1.jar \</span><br><span class="line">100</span><br></pre></td></tr></table></figure>
<p><img src="10%E3%80%81%E5%AE%98%E6%96%B9%E7%A8%8B%E5%BA%8F%E8%BF%90%E8%A1%8C%E7%BB%93%E6%9E%9C.png" alt="官方程序运行结果"></p>
</li>
<li><p>启动spark shell</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">/opt/module/spark/bin/spark-shell \</span><br><span class="line">--master spark://hadoop102:7077 \</span><br><span class="line">--executor-memory 1g \</span><br><span class="line">--total-executor-cores 2</span><br></pre></td></tr></table></figure>
<p>参数：–master spark://hadoop102:7077指定要连接的集群的master<br>执行WordCount程序</p>
<figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">scala&gt;sc.textFile(<span class="string">&quot;input&quot;</span>).flatMap(_.split(<span class="string">&quot; &quot;</span>)).map((_,<span class="number">1</span>)).reduceByKey(_+_).collect</span><br><span class="line">res0: <span class="type">Array</span>[(<span class="type">String</span>, <span class="type">Int</span>)] = <span class="type">Array</span>((hadoop,<span class="number">6</span>), (oozie,<span class="number">3</span>), (spark,<span class="number">3</span>), (hive,<span class="number">3</span>), (atguigu,<span class="number">3</span>), (hbase,<span class="number">6</span>))</span><br><span class="line"></span><br><span class="line">scala&gt;</span><br></pre></td></tr></table></figure>
<h3 id="JobHistoryServer配置"><a href="#JobHistoryServer配置" class="headerlink" title="JobHistoryServer配置"></a>JobHistoryServer配置</h3></li>
<li><p>修改spark-default.conf.template名称</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ mv spark-defaults.conf.template spark-defaults.conf</span><br></pre></td></tr></table></figure></li>
<li><p>修改spark-default.conf文件，开启Log：<br><font color=red size=3><strong>注意：HDFS上的目录需要提前存在。</strong></font></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ vi spark-defaults.conf</span><br><span class="line">spark.eventLog.enabled           true</span><br><span class="line">spark.eventLog.dir               hdfs://hadoop102:9000/directory</span><br><span class="line"></span><br><span class="line">[atguigu@hadoop102 hadoop]$ hadoop fs –mkdir /directory</span><br></pre></td></tr></table></figure></li>
<li><p>修改spark-env.sh文件，添加如下配置：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ vi spark-env.sh</span><br><span class="line"></span><br><span class="line">export SPARK_HISTORY_OPTS=&quot;-Dspark.history.ui.port=18080 </span><br><span class="line">-Dspark.history.retainedApplications=30 </span><br><span class="line">-Dspark.history.fs.logDirectory=hdfs://hadoop102:9000/directory&quot;</span><br></pre></td></tr></table></figure>
<p><font color=red size=3><strong>参数描述：</strong></font></p>
</li>
</ul>
<ol>
<li>spark.eventLog.dir：Application在运行过程中所有的信息均记录在该属性指定的路径下； </li>
<li>spark.history.ui.port=18080  WEBUI访问的端口号为18080</li>
<li>spark.history.fs.logDirectory=hdfs://hadoop102:9000/directory  配置了该属性后，在start-history-server.sh时就无需再显式的指定路径，Spark History Server页面只展示该指定路径下的信息</li>
<li>spark.history.retainedApplications=30指定保存Application历史记录的个数，如果超过这个值，旧的应用程序信息将被删除，这个是内存中的应用数，而不是页面上显示的应用数。</li>
</ol>
<ul>
<li>分发配置文件<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ xsync spark-defaults.conf</span><br><span class="line">[atguigu@hadoop102 conf]$ xsync spark-env.sh</span><br></pre></td></tr></table></figure></li>
<li>启动历史服务<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 spark]$ sbin/start-history-server.sh</span><br></pre></td></tr></table></figure></li>
<li>再次执行任务<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 spark]$ bin/spark-submit \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--master spark://hadoop102:7077 \</span><br><span class="line">--executor-memory 1G \</span><br><span class="line">--total-executor-cores 2 \</span><br><span class="line">./examples/jars/spark-examples_2.11-2.1.1.jar \</span><br><span class="line">100</span><br></pre></td></tr></table></figure></li>
<li>查看历史服务<br>hadoop102:18080<br><img src="11%E3%80%81%E6%9F%A5%E7%9C%8B%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1.png" alt="查看历史服务"></li>
</ul>
<h3 id="HA配置"><a href="#HA配置" class="headerlink" title="HA配置"></a>HA配置</h3><p><img src="12%E3%80%81HA%E6%9E%B6%E6%9E%84%E5%9B%BE.png" alt="HA架构图"></p>
<ul>
<li><p>zookeeper正常安装并启动</p>
</li>
<li><p>修改spark-env.sh文件添加如下配置：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ vi spark-env.sh</span><br></pre></td></tr></table></figure>
<p>注释掉如下内容：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">SPARK_MASTER_HOST=hadoop102</span></span><br><span class="line"><span class="meta">#</span><span class="bash">SPARK_MASTER_PORT=7077</span></span><br><span class="line">添加上如下内容：</span><br><span class="line">export SPARK_DAEMON_JAVA_OPTS=&quot;</span><br><span class="line">-Dspark.deploy.recoveryMode=ZOOKEEPER </span><br><span class="line">-Dspark.deploy.zookeeper.url=hadoop102,hadoop103,hadoop104 </span><br><span class="line">-Dspark.deploy.zookeeper.dir=/spark&quot;</span><br></pre></td></tr></table></figure></li>
<li><p>分发配置文件</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 conf]$ xsync spark-env.sh</span><br></pre></td></tr></table></figure></li>
<li><p>在hadoop102上启动全部节点</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop102 spark]$ sbin/start-all.sh</span><br></pre></td></tr></table></figure></li>
<li><p>在hadoop103上单独启动master节点【<font color=red size=3><strong>注意是在hadoop103上</strong></font>】</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[atguigu@hadoop103 spark]$ sbin/start-master.sh</span><br></pre></td></tr></table></figure></li>
<li><p>spark HA集群访问</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">/opt/module/spark/bin/spark-shell \</span><br><span class="line">--master spark://hadoop102:7077,hadoop103:7077 \</span><br><span class="line">--executor-memory 2g \</span><br><span class="line">--total-executor-cores 2</span><br></pre></td></tr></table></figure>
<h2 id="Yarn模式（重点）"><a href="#Yarn模式（重点）" class="headerlink" title="Yarn模式（重点）"></a>Yarn模式（重点）</h2><h3 id="概述-2"><a href="#概述-2" class="headerlink" title="概述"></a>概述</h3><p>Spark客户端直接连接Yarn，不需要额外构建Spark集群。有yarn-client和yarn-cluster两种模式，<font color=red size=3><strong>主要区别在于：Driver程序的运行节点。</strong></font></p>
</li>
<li><p>yarn-client：Driver程序运行在客户端，适用于交互、调试，希望立即看到app的输出</p>
</li>
<li><p>yarn-cluster：Driver程序运行在由RM（ResourceManager）启动的AP（APPMaster）适用于生产环境。<br><img src="30%E3%80%81Yarn%E8%BF%90%E8%A1%8C%E6%A8%A1%E5%BC%8F.png" alt="Yarn运行模式"><br><img src="31%E3%80%81YarnClient%E6%A8%A1%E5%BC%8F%E4%BB%BB%E5%8A%A1%E6%8F%90%E4%BA%A4%E6%B5%81%E7%A8%8B.png" alt="YarnClient模式任务提交流程"></p>
<div class="note primary flat"><p>申请资源：ApplicationMaster不清楚哪些节点的资源可以使用，所以需要向RM申请资源</p>
</div>

</li>
</ul>
<h3 id="安装使用-2"><a href="#安装使用-2" class="headerlink" title="安装使用"></a>安装使用</h3><ul>
<li>修改hadoop配置文件yarn-site.xml,添加如下内容：<div class="note primary flat"><p>因为测试环境虚拟机内存较少，防止执行过程进行被意外杀死，做如下配置</p>
</div>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 hadoop]$ vi yarn-site.xml</span><br><span class="line">        <span class="comment">&lt;!--是否启动一个线程检查每个任务正使用的物理内存量，如果任务超出分配值，则直接将其杀掉，默认是true --&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.pmem-check-enabled<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="comment">&lt;!--是否启动一个线程检查每个任务正使用的虚拟内存量，如果任务超出分配值，则直接将其杀掉，默认是true --&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.vmem-check-enabled<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure></li>
<li>修改spark-env.sh，添加如下配置：<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 conf]$ vi spark-env.sh</span><br><span class="line">YARN_CONF_DIR=/opt/module/hadoop-2.7.2/etc/hadoop</span><br></pre></td></tr></table></figure></li>
<li>分发配置文件<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 conf]$ xsync /opt/module/hadoop-2.7.2/etc/hadoop/yarn-site.xml</span><br><span class="line">[caixianquan@hadoop102 conf]$ xsync spark-env.sh</span><br></pre></td></tr></table></figure></li>
<li>执行一个程序<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ bin/spark-submit \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--master yarn \</span><br><span class="line">--deploy-mode client \</span><br><span class="line">./examples/jars/spark-examples_2.11-2.1.1.jar \</span><br><span class="line">100</span><br></pre></td></tr></table></figure>
<font color=red size=3><strong>注意：在提交任务之前需启动HDFS以及YARN集群。</strong></font></li>
</ul>
<h3 id="日志查看"><a href="#日志查看" class="headerlink" title="日志查看"></a>日志查看</h3><ul>
<li>修改配置文件spark-defaults.conf<br>添加如下内容：<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">spark.yarn.historyServer.address=hadoop102:18080</span><br><span class="line">spark.history.ui.port=18080</span><br></pre></td></tr></table></figure></li>
<li>重启spark历史服务<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ sbin/stop-history-server.sh </span><br><span class="line">stopping org.apache.spark.deploy.history.HistoryServer</span><br><span class="line">[caixianquan@hadoop102 spark]$ sbin/start-history-server.sh </span><br><span class="line">starting org.apache.spark.deploy.history.HistoryServer, logging to /opt/module/spark/logs/spark-atguigu-org.apache.spark.deploy.history.HistoryServer-1-hadoop102.out</span><br></pre></td></tr></table></figure></li>
<li>提交任务到Yarn执行<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[caixianquan@hadoop102 spark]$ bin/spark-submit \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--master yarn \</span><br><span class="line">--deploy-mode client \</span><br><span class="line">./examples/jars/spark-examples_2.11-2.1.1.jar \</span><br><span class="line">100</span><br></pre></td></tr></table></figure></li>
<li>Web页面查看日志<br><img src="32%E3%80%81web%E9%A1%B5%E9%9D%A2%E6%9F%A5%E7%9C%8B%E6%97%A5%E5%BF%97.png" alt="web页面查看日志"><img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/32%E3%80%81web%E9%A1%B5%E9%9D%A2%E6%9F%A5%E7%9C%8B%E6%97%A5%E5%BF%971.png" class="">

</li>
</ul>
<h1 id="案例实操"><a href="#案例实操" class="headerlink" title="案例实操"></a>案例实操</h1><h2 id="编写WordCount程序"><a href="#编写WordCount程序" class="headerlink" title="编写WordCount程序"></a>编写WordCount程序</h2><h3 id="创建一个Maven项目WordCount并导入依赖"><a href="#创建一个Maven项目WordCount并导入依赖" class="headerlink" title="创建一个Maven项目WordCount并导入依赖"></a>创建一个Maven项目WordCount并导入依赖</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependencies</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.apache.spark<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spark-core_2.11<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>2.1.1<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependencies</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">build</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">finalName</span>&gt;</span>WordCount<span class="tag">&lt;/<span class="name">finalName</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">plugins</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">plugin</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>net.alchim31.maven<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>scala-maven-plugin<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">version</span>&gt;</span>3.4.6<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">executions</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;<span class="name">execution</span>&gt;</span></span><br><span class="line">                       <span class="tag">&lt;<span class="name">goals</span>&gt;</span></span><br><span class="line">                          <span class="tag">&lt;<span class="name">goal</span>&gt;</span>compile<span class="tag">&lt;/<span class="name">goal</span>&gt;</span></span><br><span class="line">                          <span class="tag">&lt;<span class="name">goal</span>&gt;</span>testCompile<span class="tag">&lt;/<span class="name">goal</span>&gt;</span></span><br><span class="line">                       <span class="tag">&lt;/<span class="name">goals</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;/<span class="name">execution</span>&gt;</span></span><br><span class="line">                 <span class="tag">&lt;/<span class="name">executions</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;/<span class="name">plugin</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">plugins</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">build</span>&gt;</span></span><br></pre></td></tr></table></figure>
<div class="note primary flat"><p>注意：如果maven版本为3.2.x，插件下载报错，那么修改插件版本为3.3.2</p>
</div>
<h3 id="编写代码"><a href="#编写代码" class="headerlink" title="编写代码"></a>编写代码</h3><figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> com.atguigu</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.spark.&#123;<span class="type">SparkConf</span>, <span class="type">SparkContext</span>&#125;</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">object</span> <span class="title">WordCount</span></span>&#123;</span><br><span class="line"></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">main</span></span>(args: <span class="type">Array</span>[<span class="type">String</span>]): <span class="type">Unit</span> = &#123;</span><br><span class="line"></span><br><span class="line"><span class="comment">//1.创建SparkConf并设置App名称</span></span><br><span class="line">    <span class="keyword">val</span> conf = <span class="keyword">new</span> <span class="type">SparkConf</span>().setAppName(<span class="string">&quot;WC&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment">//2.创建SparkContext，该对象是提交Spark App的入口</span></span><br><span class="line">    <span class="keyword">val</span> sc = <span class="keyword">new</span> <span class="type">SparkContext</span>(conf)</span><br><span class="line"></span><br><span class="line">    <span class="comment">//3.使用sc创建RDD并执行相应的transformation和action</span></span><br><span class="line">    sc.textFile(args(<span class="number">0</span>)).flatMap(_.split(<span class="string">&quot; &quot;</span>)).map((_, <span class="number">1</span>)).reduceByKey(_+_, <span class="number">1</span>).sortBy(_._2, <span class="literal">false</span>).saveAsTextFile(args(<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment">//4.关闭连接</span></span><br><span class="line">    sc.stop()</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="打包插件"><a href="#打包插件" class="headerlink" title="打包插件"></a>打包插件</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">plugin</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.apache.maven.plugins<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>maven-assembly-plugin<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">version</span>&gt;</span>3.0.0<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;<span class="name">archive</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;<span class="name">manifest</span>&gt;</span></span><br><span class="line">                            <span class="tag">&lt;<span class="name">mainClass</span>&gt;</span>WordCount<span class="tag">&lt;/<span class="name">mainClass</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;/<span class="name">manifest</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;/<span class="name">archive</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;<span class="name">descriptorRefs</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;<span class="name">descriptorRef</span>&gt;</span>jar-with-dependencies<span class="tag">&lt;/<span class="name">descriptorRef</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;/<span class="name">descriptorRefs</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">executions</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;<span class="name">execution</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;<span class="name">id</span>&gt;</span>make-assembly<span class="tag">&lt;/<span class="name">id</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;<span class="name">phase</span>&gt;</span>package<span class="tag">&lt;/<span class="name">phase</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;<span class="name">goals</span>&gt;</span></span><br><span class="line">                            <span class="tag">&lt;<span class="name">goal</span>&gt;</span>single<span class="tag">&lt;/<span class="name">goal</span>&gt;</span></span><br><span class="line">                        <span class="tag">&lt;/<span class="name">goals</span>&gt;</span></span><br><span class="line">                    <span class="tag">&lt;/<span class="name">execution</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;/<span class="name">executions</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">plugin</span>&gt;</span></span><br></pre></td></tr></table></figure>
<h3 id="打包到集群测试"><a href="#打包到集群测试" class="headerlink" title="打包到集群测试"></a>打包到集群测试</h3><h4 id="通过Project-Structure方式打包"><a href="#通过Project-Structure方式打包" class="headerlink" title="通过Project Structure方式打包"></a>通过Project Structure方式打包</h4><p><font color=red size=3><strong>以下为示例</strong></font><br><img src="13%E3%80%81Project-Structure%E6%96%B9%E5%BC%8F%E6%89%93%E5%8C%85.png" alt="Project Structure方式打包"></p>
<ol>
<li>如下图所示，选中需要打包的模块，右键点击，即会出现如下图所示的内容，点击Open Module Settings。<br><img src="22%E3%80%81Open-Module-Settings.png" alt="Open Module Settings"><br><img src="14%E3%80%81Project-Structure%E9%A1%B5%E9%9D%A2.png" alt="Project-Structure页面"></li>
<li>打jar包有两种，一种是依赖jar包，Empty选项，另一种是可执行jar包，第二个选项：from modules with dependencies，会一起打包所需要的依赖jar包<br><img src="15%E3%80%81%E6%89%93%E5%8C%85%E6%89%80%E9%9C%80%E8%A6%81%E7%9A%84%E4%BE%9D%E8%B5%96jar%E5%8C%85.png" alt="打包所需要的依赖jar包"><br><img src="16%E3%80%81%E8%AE%BE%E7%BD%AE%E4%B8%BB%E7%B1%BB.png" alt="设置主类"></li>
<li>选择需要打jar的module<br><img src="17%E3%80%81%E9%80%89%E6%8B%A9%E9%9C%80%E8%A6%81%E6%89%93jar%E7%9A%84module.png" alt="选择需要打jar的module"></li>
<li>选择主类<br><img src="18%E3%80%81%E9%80%89%E6%8B%A9%E4%B8%BB%E7%B1%BB.png" alt="选择主类"><img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/19%E3%80%81%E9%80%89%E6%8B%A9%E4%B8%BB%E7%B1%BB1.png" class="">
<img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/20%E3%80%81%E9%80%89%E6%8B%A9%E4%B8%BB%E7%B1%BB2.png" class=""></li>
<li>将一些资源配置文件放在resources文件夹中<br><img src="21%E3%80%81%E8%B5%84%E6%BA%90%E9%85%8D%E7%BD%AE%E4%BD%8D%E7%BD%AE.png" alt="example"><img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/23%E3%80%81%E6%9F%A5%E7%9C%8B%E8%B5%84%E6%BA%90%E9%85%8D%E7%BD%AE%E4%BD%8D%E7%BD%AE.png" class=""></li>
<li>点击OK，点击Apply，点击OK即可</li>
<li>配置环境之后就可以开始打jar包，如下<br><img src="24%E3%80%81%E5%BC%80%E5%A7%8B%E6%89%93jar%E5%8C%85.png" alt="开始打jar包"><img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/25%E3%80%81%E5%BC%80%E5%A7%8B%E6%89%93jar%E5%8C%851.png" class=""></li>
<li>选择build，构建，成功会有如下提示<br><img src="26%E3%80%81%E6%89%93%E5%8C%85%E6%88%90%E5%8A%9F.png" alt="打包成功"></li>
<li>这个文件夹就包含了程序所需的所有依赖jar包<br><img src="27%E3%80%81%E6%9F%A5%E7%9C%8B%E7%94%9F%E6%88%90%E7%9A%84%E7%9B%AE%E5%BD%95.png" alt="查看生成的目录"><img src="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/28%E3%80%81%E6%9F%A5%E7%9C%8B%E7%94%9F%E6%88%90%E7%9A%84%E7%9B%AE%E5%BD%951.png" class=""></li>
<li>将该文件夹放入到linux中<br><img src="29%E3%80%81%E5%B0%86jar%E5%8C%85%E6%94%BE%E5%85%A5linux%E4%B8%AD.png" alt="将jar包放入linux中"></li>
</ol>
<h4 id="在yarn上运行程序"><a href="#在yarn上运行程序" class="headerlink" title="在yarn上运行程序"></a>在yarn上运行程序</h4><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">bin/spark-submit \</span><br><span class="line">--class WordCount \</span><br><span class="line">--master spark://hadoop102:7077 \</span><br><span class="line">WordCount.jar \</span><br><span class="line">/word.txt \</span><br><span class="line">/out</span><br></pre></td></tr></table></figure>
<h2 id="本地调试"><a href="#本地调试" class="headerlink" title="本地调试"></a>本地调试</h2><ul>
<li>本地Spark程序调试需要使用local提交模式，即将本机当做运行环境，Master和Worker都为本机。运行时直接加断点调试即可。如下：<br>创建SparkConf的时候设置额外属性，表明本地执行：<figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">val</span> conf = <span class="keyword">new</span> <span class="type">SparkConf</span>().setAppName(<span class="string">&quot;WC&quot;</span>).setMaster(<span class="string">&quot;local[*]&quot;</span>)</span><br></pre></td></tr></table></figure></li>
<li>如果本机操作系统是windows，如果在程序中使用了hadoop相关的东西，比如写入文件到HDFS，则会遇到如下异常：<br><img src="33%E3%80%81%E6%9C%AC%E5%9C%B0%E8%BF%90%E8%A1%8Cspark%E7%A8%8B%E5%BA%8F%E6%8A%A5%E9%94%99.png" alt="本地运行spark程序报错"></li>
<li>出现这个问题的原因，并不是程序的错误，而是用到了hadoop相关的服务，解决办法是将附加里面的hadoop-common-bin-2.7.3-x64.zip解压到任意目录。<br><img src="34%E3%80%81hadoop%E7%9B%AE%E5%BD%95.png" alt="hadoop目录"></li>
<li>在IDEA中配置Run Configuration，添加HADOOP_HOME变量<br><img src="35%E3%80%81idea%E6%B7%BB%E5%8A%A0HADOOP_HOME%E5%8F%98%E9%87%8F.png" alt="idea添加HADOOP_HOME变量"></li>
</ul>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">Cai XianQuan</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/">https://xquan123.gitee.io/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://xquan123.gitee.io" target="_blank">Cquang博客</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/blog/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a><a class="post-meta__tags" href="/blog/tags/spark/">spark</a><a class="post-meta__tags" href="/blog/tags/spark%E8%BF%90%E8%A1%8C%E6%A8%A1%E5%BC%8F/">spark运行模式</a></div><div class="post_share"><div class="social-share" data-image="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90%E9%A6%96%E9%A1%B5.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSparkCore-%E4%BA%8C/"><img class="prev-cover" src="/blog/img/default_cover.jpg" onerror="onerror=null;src='/blog/img/404.jpg'"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">尚硅谷大数据之SparkCore(二)</div></div></a></div><div class="next-post pull-right"><a href="/blog/2020/07/19/Python%E5%9F%BA%E7%A1%80/"><img class="next-cover" src="/blog/2020/07/19/Python%E5%9F%BA%E7%A1%80/Python%E5%9F%BA%E7%A1%80%E9%A6%96%E9%A1%B5.jpg" onerror="onerror=null;src='/blog/img/404.jpg'"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">Python基础</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span> 相关推荐</span></div><div class="relatedPosts-list"><div><a href="/blog/2020/12/08/kafka入门学习/" title="kafka入门学习"><img class="cover" src="/blog/2020/12/08/kafka%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/kafka%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-12-08</div><div class="title">kafka入门学习</div></div></a></div><div><a href="/blog/2020/12/13/Flink入门学习/" title="Flink入门学习"><img class="cover" src="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-12-13</div><div class="title">Flink入门学习</div></div></a></div></div></div></div><div class="aside_content" id="aside_content"><div class="card-widget card-info"><div class="card-content"><div class="card-info-avatar is-center"><img class="avatar-img" src="/blog/img/%E5%A4%B4%E5%83%8F.gif" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="avatar"/><div class="author-info__name">Cai XianQuan</div><div class="author-info__description"></div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">22</div></a></div><div class="card-info-data-item is-center"><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">33</div></a></div><div class="card-info-data-item is-center"><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div></div><a class="button--animated" id="card-info-btn" href="javascript:;"><i class="fab fa-bookmark"></i><span>加入书签</span></a></div></div><div class="card-widget card-announcement"><div class="card-content"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content">一花一世界,一木一菩提╮(￣▽ ￣)╭</div></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="card-content"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#Spark%E6%A6%82%E8%BF%B0"><span class="toc-number">1.</span> <span class="toc-text">Spark概述</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%80%E4%B9%88%E6%98%AFSpark"><span class="toc-number">1.1.</span> <span class="toc-text">什么是Spark</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9A%E4%B9%89"><span class="toc-number">1.1.1.</span> <span class="toc-text">定义</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%8E%86%E5%8F%B2"><span class="toc-number">1.1.2.</span> <span class="toc-text">历史</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hadoop-MR%E6%A1%86%E6%9E%B6%E5%92%8CSpark%E6%A1%86%E6%9E%B6"><span class="toc-number">1.1.3.</span> <span class="toc-text">Hadoop MR框架和Spark框架</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Spark%E5%86%85%E7%BD%AE%E6%A8%A1%E5%9D%97"><span class="toc-number">1.2.</span> <span class="toc-text">Spark内置模块</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Spark%E7%89%B9%E7%82%B9"><span class="toc-number">1.3.</span> <span class="toc-text">Spark特点</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#Spark%E8%BF%90%E8%A1%8C%E6%A8%A1%E5%BC%8F"><span class="toc-number">2.</span> <span class="toc-text">Spark运行模式</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#Spark%E5%AE%89%E8%A3%85%E5%9C%B0%E5%9D%80"><span class="toc-number">2.1.</span> <span class="toc-text">Spark安装地址</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%87%8D%E8%A6%81%E8%A7%92%E8%89%B2"><span class="toc-number">2.2.</span> <span class="toc-text">重要角色</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Driver%EF%BC%88%E9%A9%B1%E5%8A%A8%E5%99%A8%EF%BC%89%E3%80%90%E7%AE%A1%E7%90%86%E8%80%85%E3%80%91"><span class="toc-number">2.2.1.</span> <span class="toc-text">Driver（驱动器）【管理者】</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Executor%EF%BC%88%E6%89%A7%E8%A1%8C%E5%99%A8%EF%BC%89"><span class="toc-number">2.2.2.</span> <span class="toc-text">Executor（执行器）</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Local%E6%A8%A1%E5%BC%8F"><span class="toc-number">2.3.</span> <span class="toc-text">Local模式</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A6%82%E8%BF%B0"><span class="toc-number">2.3.1.</span> <span class="toc-text">概述</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8"><span class="toc-number">2.3.2.</span> <span class="toc-text">安装使用</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%BF%90%E8%A1%8CWordCount%E7%A8%8B%E5%BA%8F"><span class="toc-number">2.3.3.</span> <span class="toc-text">运行WordCount程序</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Standalone%E6%A8%A1%E5%BC%8F"><span class="toc-number">2.4.</span> <span class="toc-text">Standalone模式</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A6%82%E8%BF%B0-1"><span class="toc-number">2.4.1.</span> <span class="toc-text">概述</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8-1"><span class="toc-number">2.4.2.</span> <span class="toc-text">安装使用</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#JobHistoryServer%E9%85%8D%E7%BD%AE"><span class="toc-number">2.4.3.</span> <span class="toc-text">JobHistoryServer配置</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HA%E9%85%8D%E7%BD%AE"><span class="toc-number">2.4.4.</span> <span class="toc-text">HA配置</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Yarn%E6%A8%A1%E5%BC%8F%EF%BC%88%E9%87%8D%E7%82%B9%EF%BC%89"><span class="toc-number">2.5.</span> <span class="toc-text">Yarn模式（重点）</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A6%82%E8%BF%B0-2"><span class="toc-number">2.5.1.</span> <span class="toc-text">概述</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8-2"><span class="toc-number">2.5.2.</span> <span class="toc-text">安装使用</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%97%A5%E5%BF%97%E6%9F%A5%E7%9C%8B"><span class="toc-number">2.5.3.</span> <span class="toc-text">日志查看</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E6%A1%88%E4%BE%8B%E5%AE%9E%E6%93%8D"><span class="toc-number">3.</span> <span class="toc-text">案例实操</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%BC%96%E5%86%99WordCount%E7%A8%8B%E5%BA%8F"><span class="toc-number">3.1.</span> <span class="toc-text">编写WordCount程序</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%9B%E5%BB%BA%E4%B8%80%E4%B8%AAMaven%E9%A1%B9%E7%9B%AEWordCount%E5%B9%B6%E5%AF%BC%E5%85%A5%E4%BE%9D%E8%B5%96"><span class="toc-number">3.1.1.</span> <span class="toc-text">创建一个Maven项目WordCount并导入依赖</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%BC%96%E5%86%99%E4%BB%A3%E7%A0%81"><span class="toc-number">3.1.2.</span> <span class="toc-text">编写代码</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%89%93%E5%8C%85%E6%8F%92%E4%BB%B6"><span class="toc-number">3.1.3.</span> <span class="toc-text">打包插件</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%89%93%E5%8C%85%E5%88%B0%E9%9B%86%E7%BE%A4%E6%B5%8B%E8%AF%95"><span class="toc-number">3.1.4.</span> <span class="toc-text">打包到集群测试</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%80%9A%E8%BF%87Project-Structure%E6%96%B9%E5%BC%8F%E6%89%93%E5%8C%85"><span class="toc-number">3.1.4.1.</span> <span class="toc-text">通过Project Structure方式打包</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%9C%A8yarn%E4%B8%8A%E8%BF%90%E8%A1%8C%E7%A8%8B%E5%BA%8F"><span class="toc-number">3.1.4.2.</span> <span class="toc-text">在yarn上运行程序</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%9C%AC%E5%9C%B0%E8%B0%83%E8%AF%95"><span class="toc-number">3.2.</span> <span class="toc-text">本地调试</span></a></li></ol></li></ol></div></div></div><div class="card-widget card-recent-post"><div class="card-content"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/" title="框架套路解算法"><img src="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="框架套路解算法"/></a><div class="content"><a class="title" href="/blog/2021/01/29/%E6%A1%86%E6%9E%B6%E5%A5%97%E8%B7%AF%E8%A7%A3%E7%AE%97%E6%B3%95/" title="框架套路解算法">框架套路解算法</a><time datetime="2021-01-29T07:37:55.000Z" title="发表于 2021-01-29 15:37:55">2021-01-29</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/" title="shell脚本学习"><img src="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="shell脚本学习"/></a><div class="content"><a class="title" href="/blog/2021/01/07/shell%E8%84%9A%E6%9C%AC%E5%AD%A6%E4%B9%A0/" title="shell脚本学习">shell脚本学习</a><time datetime="2021-01-07T03:29:18.000Z" title="发表于 2021-01-07 11:29:18">2021-01-07</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/" title="Linux问题集锦"><img src="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6%E9%A6%96%E9%A1%B5.png" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="Linux问题集锦"/></a><div class="content"><a class="title" href="/blog/2020/12/28/Linux%E9%97%AE%E9%A2%98%E9%9B%86%E9%94%A6/" title="Linux问题集锦">Linux问题集锦</a><time datetime="2020-12-28T01:56:38.000Z" title="发表于 2020-12-28 09:56:38">2020-12-28</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/" title="常用的Linux命令集"><img src="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86%E9%A6%96%E9%A1%B5.jpg" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="常用的Linux命令集"/></a><div class="content"><a class="title" href="/blog/2020/12/13/%E5%B8%B8%E7%94%A8%E7%9A%84Linux%E5%91%BD%E4%BB%A4%E9%9B%86/" title="常用的Linux命令集">常用的Linux命令集</a><time datetime="2020-12-13T14:32:23.000Z" title="发表于 2020-12-13 22:32:23">2020-12-13</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/" title="Flink入门学习"><img src="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0%E9%A6%96%E9%A1%B5.png" onerror="this.onerror=null;this.src='/blog/img/404.jpg'" alt="Flink入门学习"/></a><div class="content"><a class="title" href="/blog/2020/12/13/Flink%E5%85%A5%E9%97%A8%E5%AD%A6%E4%B9%A0/" title="Flink入门学习">Flink入门学习</a><time datetime="2020-12-13T10:21:57.000Z" title="发表于 2020-12-13 18:21:57">2020-12-13</time></div></div></div></div></div></div></div></main><footer id="footer" style="background-image: url(/blog/2020/08/08/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90-%E4%B8%80/%E5%B0%9A%E7%A1%85%E8%B0%B7%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BSpark%E5%9F%BA%E7%A1%80%E8%A7%A3%E6%9E%90%E9%A6%96%E9%A1%B5.jpg)"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By Cai XianQuan</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">Hi,  welcome  to  my  <a  target="_blank" rel="noopener" href="https://www.caixianquan.tk">blog</a>!</div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"><div id="local-hits"></div><div id="local-stats"><div class="local-search-stats__hr" id="hr"><span>由</span> <a target="_blank" rel="noopener" href="https://github.com/wzpan/hexo-generator-search" style="color:#49B1F5;">hexo-generator-search</a>
 <span>提供支持</span></div></div></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js"></script><script src="/blog/js/utils.js"></script><script src="/blog/js/main.js"></script><script src="/blog/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/blog/js/search/local-search.js"></script><div class="js-pjax"><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><script src="/js/bookmark.js"></script><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="true"></script></div></body></html>